{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_path = \"/Users/mac/Downloads/datasets/cmrc2018/\"\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import Dataset\n",
    "import json\n",
    "\n",
    "class CMRC2018(Dataset):\n",
    "    def __init__(self, data_file):\n",
    "        self.data = self.load_data(data_file)\n",
    "    \n",
    "    def load_data(self, data_file):\n",
    "        Data = {}\n",
    "        with open(data_file, 'r', encoding='utf-8') as f:\n",
    "            json_data = json.load(f)\n",
    "            idx = 0\n",
    "            for article in json_data['data']:\n",
    "                title = article['title']\n",
    "                context = article['paragraphs'][0]['context']\n",
    "                for question in article['paragraphs'][0]['qas']:\n",
    "                    q_id = question['id']\n",
    "                    ques = question['question']\n",
    "                    text = [ans['text'] for ans in question['answers']]\n",
    "                    answer_start = [ans['answer_start'] for ans in question['answers']]\n",
    "                    Data[idx] = {\n",
    "                        'id': q_id,\n",
    "                        'title': title,\n",
    "                        'context': context, \n",
    "                        'question': ques,\n",
    "                        'answers': {\n",
    "                            'text': text,\n",
    "                            'answer_start': answer_start\n",
    "                        }\n",
    "                    }\n",
    "                    idx += 1\n",
    "        return Data\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.data)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        return self.data[idx]\n",
    "\n",
    "train_data = CMRC2018(file_path + 'cmrc2018_train.json')\n",
    "valid_data = CMRC2018(file_path + 'cmrc2018_dev.json')\n",
    "test_data = CMRC2018(file_path + 'cmrc2018_trial.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train set size: 10142\n",
      "valid set size: 3219\n",
      "test set size: 1002\n",
      "{'id': 'DEV_0_QUERY_0', 'title': '战国无双3', 'context': '《战国无双3》（）是由光荣和ω-force开发的战国无双系列的正统第三续作。本作以三大故事为主轴，分别是以武田信玄等人为主的《关东三国志》，织田信长等人为主的《战国三杰》，石田三成等人为主的《关原的年轻武者》，丰富游戏内的剧情。此部份专门介绍角色，欲知武器情报、奥义字或擅长攻击类型等，请至战国无双系列1.由于乡里大辅先生因故去世，不得不寻找其他声优接手。从猛将传 and Z开始。2.战国无双 编年史的原创男女主角亦有专属声优。此模式是任天堂游戏谜之村雨城改编的新增模式。本作中共有20张战场地图（不含村雨城），后来发行的猛将传再新增3张战场地图。但游戏内战役数量繁多，部分地图会有兼用的状况，战役虚实则是以光荣发行的2本「战国无双3 人物真书」内容为主，以下是相关介绍。（注：前方加☆者为猛将传新增关卡及地图。）合并本篇和猛将传的内容，村雨城模式剔除，战国史模式可直接游玩。主打两大模式「战史演武」&「争霸演武」。系列作品外传作品', 'question': '《战国无双3》是由哪两个公司合作开发的？', 'answers': {'text': ['光荣和ω-force', '光荣和ω-force', '光荣和ω-force'], 'answer_start': [11, 11, 11]}}\n"
     ]
    }
   ],
   "source": [
    "print(f'train set size: {len(train_data)}')\n",
    "print(f'valid set size: {len(valid_data)}')\n",
    "print(f'test set size: {len(test_data)}')\n",
    "print(next(iter(valid_data)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoTokenizer\n",
    "\n",
    "checkpoint = 'bert-base-chinese'\n",
    "tokenizer = AutoTokenizer.from_pretrained(checkpoint)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "context = train_data[0][\"context\"]\n",
    "question = train_data[0][\"question\"]\n",
    "\n",
    "inputs = tokenizer(\n",
    "    question,\n",
    "    context,\n",
    "    max_length=300,\n",
    "    truncation=\"only_second\",\n",
    "    stride=50,\n",
    "    return_overflowing_tokens=True,\n",
    "    return_offsets_mapping=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-07-01 09:15:58.454519: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CLS] 范 廷 颂 是 什 么 时 候 被 任 为 主 教 的 ？ [SEP] 范 廷 颂 枢 机 （ ， ） ， 圣 名 保 禄 · 若 瑟 （ ） ， 是 越 南 罗 马 天 主 教 枢 机 。 1963 年 被 任 为 主 教 ； 1990 年 被 擢 升 为 天 主 教 河 内 总 教 区 宗 座 署 理 ； 1994 年 被 擢 升 为 总 主 教 ， 同 年 年 底 被 擢 升 为 枢 机 ； 2009 年 2 月 离 世 。 范 廷 颂 于 1919 年 6 月 15 日 在 越 南 宁 平 省 天 主 教 发 艳 教 区 出 生 ； 童 年 时 接 受 良 好 教 育 后 ， 被 一 位 越 南 神 父 带 到 河 内 继 续 其 学 业 。 范 廷 颂 于 1940 年 在 河 内 大 修 道 院 完 成 神 学 学 业 。 范 廷 颂 于 1949 年 6 月 6 日 在 河 内 的 主 教 座 堂 晋 铎 ； 及 后 被 派 到 圣 女 小 德 兰 孤 儿 院 服 务 。 1950 年 代 ， 范 廷 颂 在 河 内 堂 区 创 建 移 民 接 待 中 心 以 收 容 到 河 内 避 战 的 难 民 。 1954 年 ， 法 越 战 争 结 束 ， 越 南 民 主 共 和 国 建 都 河 内 ， 当 时 很 多 天 主 教 神 职 人 员 逃 至 越 南 的 南 方 ， 但 范 廷 颂 仍 然 留 在 河 内 。 翌 年 [SEP]\n",
      "[CLS] 范 廷 颂 是 什 么 时 候 被 任 为 主 教 的 ？ [SEP] 越 战 争 结 束 ， 越 南 民 主 共 和 国 建 都 河 内 ， 当 时 很 多 天 主 教 神 职 人 员 逃 至 越 南 的 南 方 ， 但 范 廷 颂 仍 然 留 在 河 内 。 翌 年 管 理 圣 若 望 小 修 院 ； 惟 在 1960 年 因 捍 卫 修 院 的 自 由 、 自 治 及 拒 绝 政 府 在 修 院 设 政 治 课 的 要 求 而 被 捕 。 1963 年 4 月 5 日 ， 教 宗 任 命 范 廷 颂 为 天 主 教 北 宁 教 区 主 教 ， 同 年 8 月 15 日 就 任 ； 其 牧 铭 为 「 我 信 天 主 的 爱 」 。 由 于 范 廷 颂 被 越 南 政 府 软 禁 差 不 多 30 年 ， 因 此 他 无 法 到 所 属 堂 区 进 行 牧 灵 工 作 而 专 注 研 读 等 工 作 。 范 廷 颂 除 了 面 对 战 争 、 贫 困 、 被 当 局 迫 害 天 主 教 会 等 问 题 外 ， 也 秘 密 恢 复 修 院 、 创 建 女 修 会 团 体 等 。 1990 年 ， 教 宗 若 望 保 禄 二 世 在 同 年 6 月 18 日 擢 升 范 廷 颂 为 天 主 教 河 内 总 教 区 宗 座 署 理 以 填 补 该 教 区 总 主 教 的 空 缺 。 1994 年 3 月 23 日 [SEP]\n",
      "[CLS] 范 廷 颂 是 什 么 时 候 被 任 为 主 教 的 ？ [SEP] 若 望 保 禄 二 世 在 同 年 6 月 18 日 擢 升 范 廷 颂 为 天 主 教 河 内 总 教 区 宗 座 署 理 以 填 补 该 教 区 总 主 教 的 空 缺 。 1994 年 3 月 23 日 ， 范 廷 颂 被 教 宗 若 望 保 禄 二 世 擢 升 为 天 主 教 河 内 总 教 区 总 主 教 并 兼 天 主 教 谅 山 教 区 宗 座 署 理 ； 同 年 11 月 26 日 ， 若 望 保 禄 二 世 擢 升 范 廷 颂 为 枢 机 。 范 廷 颂 在 1995 年 至 2001 年 期 间 出 任 天 主 教 越 南 主 教 团 主 席 。 2003 年 4 月 26 日 ， 教 宗 若 望 保 禄 二 世 任 命 天 主 教 谅 山 教 区 兼 天 主 教 高 平 教 区 吴 光 杰 主 教 为 天 主 教 河 内 总 教 区 署 理 主 教 ； 及 至 2005 年 2 月 19 日 ， 范 廷 颂 因 获 批 辞 去 总 主 教 职 务 而 荣 休 ； 吴 光 杰 同 日 真 除 天 主 教 河 内 总 教 区 总 主 教 职 务 。 范 廷 颂 于 2009 年 2 月 22 日 清 晨 在 河 内 离 世 ， 享 年 89 岁 ； 其 葬 礼 于 同 月 26 日 上 午 在 天 主 教 河 内 总 教 区 总 主 教 座 堂 [SEP]\n",
      "[CLS] 范 廷 颂 是 什 么 时 候 被 任 为 主 教 的 ？ [SEP] 职 务 。 范 廷 颂 于 2009 年 2 月 22 日 清 晨 在 河 内 离 世 ， 享 年 89 岁 ； 其 葬 礼 于 同 月 26 日 上 午 在 天 主 教 河 内 总 教 区 总 主 教 座 堂 举 行 。 [SEP]\n"
     ]
    }
   ],
   "source": [
    "for ids in inputs[\"input_ids\"]:\n",
    "    print(tokenizer.decode(ids))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['[CLS]', '范', '廷', '颂', '是', '什', '么', '时', '候', '被', '任', '为', '主', '教', '的', '？', '[SEP]', '范', '廷', '颂', '枢', '机', '（', '，', '）', '，', '圣', '名', '保', '禄', '·', '若', '瑟', '（', '）', '，', '是', '越', '南', '罗', '马', '天', '主', '教', '枢', '机', '。', '1963', '年', '被', '任', '为', '主', '教', '；', '1990', '年', '被', '擢', '升', '为', '天', '主', '教', '河', '内', '总', '教', '区', '宗', '座', '署', '理', '；', '1994', '年', '被', '擢', '升', '为', '总', '主', '教', '，', '同', '年', '年', '底', '被', '擢', '升', '为', '枢', '机', '；', '2009', '年', '2', '月', '离', '世', '。', '范', '廷', '颂', '于', '1919', '年', '6', '月', '15', '日', '在', '越', '南', '宁', '平', '省', '天', '主', '教', '发', '艳', '教', '区', '出', '生', '；', '童', '年', '时', '接', '受', '良', '好', '教', '育', '后', '，', '被', '一', '位', '越', '南', '神', '父', '带', '到', '河', '内', '继', '续', '其', '学', '业', '。', '范', '廷', '颂', '于', '1940', '年', '在', '河', '内', '大', '修', '道', '院', '完', '成', '神', '学', '学', '业', '。', '范', '廷', '颂', '于', '1949', '年', '6', '月', '6', '日', '在', '河', '内', '的', '主', '教', '座', '堂', '晋', '铎', '；', '及', '后', '被', '派', '到', '圣', '女', '小', '德', '兰', '孤', '儿', '院', '服', '务', '。', '1950', '年', '代', '，', '范', '廷', '颂', '在', '河', '内', '堂', '区', '创', '建', '移', '民', '接', '待', '中', '心', '以', '收', '容', '到', '河', '内', '避', '战', '的', '难', '民', '。', '1954', '年', '，', '法', '越', '战', '争', '结', '束', '，', '越', '南', '民', '主', '共', '和', '国', '建', '都', '河', '内', '，', '当', '时', '很', '多', '天', '主', '教', '神', '职', '人', '员', '逃', '至', '越', '南', '的', '南', '方', '，', '但', '范', '廷', '颂', '仍', '然', '留', '在', '河', '内', '。', '翌', '年', '[SEP]']\n",
      "300\n"
     ]
    }
   ],
   "source": [
    "print(inputs.tokens())\n",
    "print(len(inputs.tokens()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_keys(['input_ids', 'token_type_ids', 'attention_mask', 'offset_mapping', 'overflow_to_sample_mapping'])\n"
     ]
    }
   ],
   "source": [
    "print(inputs.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[101, 5745, 2455, 7563, 3221, 784, 720, 3198, 952, 6158, 818, 711, 712, 3136, 4638, 8043, 102, 5745, 2455, 7563, 3364, 3322, 8020, 8024, 8021, 8024, 1760, 1399, 924, 4882, 185, 5735, 4449, 8020, 8021, 8024, 3221, 6632, 1298, 5384, 7716, 1921, 712, 3136, 3364, 3322, 511, 9155, 2399, 6158, 818, 711, 712, 3136, 8039, 8431, 2399, 6158, 3091, 1285, 711, 1921, 712, 3136, 3777, 1079, 2600, 3136, 1277, 2134, 2429, 5392, 4415, 8039, 8447, 2399, 6158, 3091, 1285, 711, 2600, 712, 3136, 8024, 1398, 2399, 2399, 2419, 6158, 3091, 1285, 711, 3364, 3322, 8039, 8170, 2399, 123, 3299, 4895, 686, 511, 5745, 2455, 7563, 754, 9915, 2399, 127, 3299, 8115, 3189, 1762, 6632, 1298, 2123, 2398, 4689, 1921, 712, 3136, 1355, 5683, 3136, 1277, 1139, 4495, 8039, 4997, 2399, 3198, 2970, 1358, 5679, 1962, 3136, 5509, 1400, 8024, 6158, 671, 855, 6632, 1298, 4868, 4266, 2372, 1168, 3777, 1079, 5326, 5330, 1071, 2110, 689, 511, 5745, 2455, 7563, 754, 9211, 2399, 1762, 3777, 1079, 1920, 934, 6887, 7368, 2130, 2768, 4868, 2110, 2110, 689, 511, 5745, 2455, 7563, 754, 8594, 2399, 127, 3299, 127, 3189, 1762, 3777, 1079, 4638, 712, 3136, 2429, 1828, 3232, 7195, 8039, 1350, 1400, 6158, 3836, 1168, 1760, 1957, 2207, 2548, 1065, 2109, 1036, 7368, 3302, 1218, 511, 8707, 2399, 807, 8024, 5745, 2455, 7563, 1762, 3777, 1079, 1828, 1277, 1158, 2456, 4919, 3696, 2970, 2521, 704, 2552, 809, 3119, 2159, 1168, 3777, 1079, 6912, 2773, 4638, 7410, 3696, 511, 9258, 2399, 8024, 3791, 6632, 2773, 751, 5310, 3338, 8024, 6632, 1298, 3696, 712, 1066, 1469, 1744, 2456, 6963, 3777, 1079, 8024, 2496, 3198, 2523, 1914, 1921, 712, 3136, 4868, 5466, 782, 1447, 6845, 5635, 6632, 1298, 4638, 1298, 3175, 8024, 852, 5745, 2455, 7563, 793, 4197, 4522, 1762, 3777, 1079, 511, 5422, 2399, 102]\n"
     ]
    }
   ],
   "source": [
    "print(inputs['input_ids'][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n"
     ]
    }
   ],
   "source": [
    "print(inputs['token_type_ids'][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n"
     ]
    }
   ],
   "source": [
    "print(inputs['attention_mask'][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[(0, 0), (0, 1), (1, 2), (2, 3), (3, 4), (4, 5), (5, 6), (6, 7), (7, 8), (8, 9), (9, 10), (10, 11), (11, 12), (12, 13), (13, 14), (14, 15), (0, 0), (0, 1), (1, 2), (2, 3), (3, 4), (4, 5), (5, 6), (6, 7), (7, 8), (8, 9), (9, 10), (10, 11), (11, 12), (12, 13), (13, 14), (14, 15), (15, 16), (16, 17), (17, 18), (18, 19), (19, 20), (20, 21), (21, 22), (22, 23), (23, 24), (24, 25), (25, 26), (26, 27), (27, 28), (28, 29), (29, 30), (30, 34), (34, 35), (35, 36), (36, 37), (37, 38), (38, 39), (39, 40), (40, 41), (41, 45), (45, 46), (46, 47), (47, 48), (48, 49), (49, 50), (50, 51), (51, 52), (52, 53), (53, 54), (54, 55), (55, 56), (56, 57), (57, 58), (58, 59), (59, 60), (60, 61), (61, 62), (62, 63), (63, 67), (67, 68), (68, 69), (69, 70), (70, 71), (71, 72), (72, 73), (73, 74), (74, 75), (75, 76), (76, 77), (77, 78), (78, 79), (79, 80), (80, 81), (81, 82), (82, 83), (83, 84), (84, 85), (85, 86), (86, 87), (87, 91), (91, 92), (92, 93), (93, 94), (94, 95), (95, 96), (96, 97), (97, 98), (98, 99), (99, 100), (100, 101), (101, 105), (105, 106), (106, 107), (107, 108), (108, 110), (110, 111), (111, 112), (112, 113), (113, 114), (114, 115), (115, 116), (116, 117), (117, 118), (118, 119), (119, 120), (120, 121), (121, 122), (122, 123), (123, 124), (124, 125), (125, 126), (126, 127), (127, 128), (128, 129), (129, 130), (130, 131), (131, 132), (132, 133), (133, 134), (134, 135), (135, 136), (136, 137), (137, 138), (138, 139), (139, 140), (140, 141), (141, 142), (142, 143), (143, 144), (144, 145), (145, 146), (146, 147), (147, 148), (148, 149), (149, 150), (150, 151), (151, 152), (152, 153), (153, 154), (154, 155), (155, 156), (156, 157), (157, 158), (158, 159), (159, 163), (163, 164), (164, 165), (165, 166), (166, 167), (167, 168), (168, 169), (169, 170), (170, 171), (171, 172), (172, 173), (173, 174), (174, 175), (175, 176), (176, 177), (177, 178), (178, 179), (179, 180), (180, 181), (181, 182), (182, 186), (186, 187), (187, 188), (188, 189), (189, 190), (190, 191), (191, 192), (192, 193), (193, 194), (194, 195), (195, 196), (196, 197), (197, 198), (198, 199), (199, 200), (200, 201), (201, 202), (202, 203), (203, 204), (204, 205), (205, 206), (206, 207), (207, 208), (208, 209), (209, 210), (210, 211), (211, 212), (212, 213), (213, 214), (214, 215), (215, 216), (216, 217), (217, 218), (218, 222), (222, 223), (223, 224), (224, 225), (225, 226), (226, 227), (227, 228), (228, 229), (229, 230), (230, 231), (231, 232), (232, 233), (233, 234), (234, 235), (235, 236), (236, 237), (237, 238), (238, 239), (239, 240), (240, 241), (241, 242), (242, 243), (243, 244), (244, 245), (245, 246), (246, 247), (247, 248), (248, 249), (249, 250), (250, 251), (251, 252), (252, 253), (253, 257), (257, 258), (258, 259), (259, 260), (260, 261), (261, 262), (262, 263), (263, 264), (264, 265), (265, 266), (266, 267), (267, 268), (268, 269), (269, 270), (270, 271), (271, 272), (272, 273), (273, 274), (274, 275), (275, 276), (276, 277), (277, 278), (278, 279), (279, 280), (280, 281), (281, 282), (282, 283), (283, 284), (284, 285), (285, 286), (286, 287), (287, 288), (288, 289), (289, 290), (290, 291), (291, 292), (292, 293), (293, 294), (294, 295), (295, 296), (296, 297), (297, 298), (298, 299), (299, 300), (300, 301), (301, 302), (302, 303), (303, 304), (304, 305), (305, 306), (306, 307), (307, 308), (308, 309), (309, 310), (0, 0)], [(0, 0), (0, 1), (1, 2), (2, 3), (3, 4), (4, 5), (5, 6), (6, 7), (7, 8), (8, 9), (9, 10), (10, 11), (11, 12), (12, 13), (13, 14), (14, 15), (0, 0), (260, 261), (261, 262), (262, 263), (263, 264), (264, 265), (265, 266), (266, 267), (267, 268), (268, 269), (269, 270), (270, 271), (271, 272), (272, 273), (273, 274), (274, 275), (275, 276), (276, 277), (277, 278), (278, 279), (279, 280), (280, 281), (281, 282), (282, 283), (283, 284), (284, 285), (285, 286), (286, 287), (287, 288), (288, 289), (289, 290), (290, 291), (291, 292), (292, 293), (293, 294), (294, 295), (295, 296), (296, 297), (297, 298), (298, 299), (299, 300), (300, 301), (301, 302), (302, 303), (303, 304), (304, 305), (305, 306), (306, 307), (307, 308), (308, 309), (309, 310), (310, 311), (311, 312), (312, 313), (313, 314), (314, 315), (315, 316), (316, 317), (317, 318), (318, 319), (319, 320), (320, 321), (321, 325), (325, 326), (326, 327), (327, 328), (328, 329), (329, 330), (330, 331), (331, 332), (332, 333), (333, 334), (334, 335), (335, 336), (336, 337), (337, 338), (338, 339), (339, 340), (340, 341), (341, 342), (342, 343), (343, 344), (344, 345), (345, 346), (346, 347), (347, 348), (348, 349), (349, 350), (350, 351), (351, 352), (352, 353), (353, 354), (354, 355), (355, 356), (356, 360), (360, 361), (361, 362), (362, 363), (363, 364), (364, 365), (365, 366), (366, 367), (367, 368), (368, 369), (369, 370), (370, 371), (371, 372), (372, 373), (373, 374), (374, 375), (375, 376), (376, 377), (377, 378), (378, 379), (379, 380), (380, 381), (381, 382), (382, 383), (383, 384), (384, 385), (385, 386), (386, 387), (387, 388), (388, 390), (390, 391), (391, 392), (392, 393), (393, 394), (394, 395), (395, 396), (396, 397), (397, 398), (398, 399), (399, 400), (400, 401), (401, 402), (402, 403), (403, 404), (404, 405), (405, 406), (406, 407), (407, 408), (408, 409), (409, 410), (410, 411), (411, 412), (412, 413), (413, 414), (414, 415), (415, 416), (416, 417), (417, 418), (418, 419), (419, 420), (420, 421), (421, 422), (422, 424), (424, 425), (425, 426), (426, 427), (427, 428), (428, 429), (429, 430), (430, 431), (431, 432), (432, 433), (433, 434), (434, 435), (435, 436), (436, 437), (437, 438), (438, 439), (439, 440), (440, 441), (441, 442), (442, 443), (443, 444), (444, 445), (445, 446), (446, 447), (447, 448), (448, 449), (449, 450), (450, 451), (451, 452), (452, 453), (453, 454), (454, 455), (455, 456), (456, 457), (457, 458), (458, 459), (459, 460), (460, 461), (461, 462), (462, 463), (463, 464), (464, 465), (465, 466), (466, 467), (467, 468), (468, 469), (469, 470), (470, 471), (471, 472), (472, 473), (473, 474), (474, 475), (475, 476), (476, 477), (477, 478), (478, 479), (479, 480), (480, 481), (481, 482), (482, 483), (483, 484), (484, 485), (485, 486), (486, 487), (487, 488), (488, 489), (489, 490), (490, 491), (491, 492), (492, 493), (493, 494), (494, 495), (495, 499), (499, 500), (500, 501), (501, 502), (502, 503), (503, 504), (504, 505), (505, 506), (506, 507), (507, 508), (508, 509), (509, 510), (510, 511), (511, 512), (512, 513), (513, 514), (514, 516), (516, 517), (517, 518), (518, 519), (519, 520), (520, 521), (521, 522), (522, 523), (523, 524), (524, 525), (525, 526), (526, 527), (527, 528), (528, 529), (529, 530), (530, 531), (531, 532), (532, 533), (533, 534), (534, 535), (535, 536), (536, 537), (537, 538), (538, 539), (539, 540), (540, 541), (541, 542), (542, 543), (543, 544), (544, 545), (545, 546), (546, 547), (547, 548), (548, 552), (552, 553), (553, 554), (554, 555), (555, 557), (557, 558), (0, 0)], [(0, 0), (0, 1), (1, 2), (2, 3), (3, 4), (4, 5), (5, 6), (6, 7), (7, 8), (8, 9), (9, 10), (10, 11), (11, 12), (12, 13), (13, 14), (14, 15), (0, 0), (503, 504), (504, 505), (505, 506), (506, 507), (507, 508), (508, 509), (509, 510), (510, 511), (511, 512), (512, 513), (513, 514), (514, 516), (516, 517), (517, 518), (518, 519), (519, 520), (520, 521), (521, 522), (522, 523), (523, 524), (524, 525), (525, 526), (526, 527), (527, 528), (528, 529), (529, 530), (530, 531), (531, 532), (532, 533), (533, 534), (534, 535), (535, 536), (536, 537), (537, 538), (538, 539), (539, 540), (540, 541), (541, 542), (542, 543), (543, 544), (544, 545), (545, 546), (546, 547), (547, 548), (548, 552), (552, 553), (553, 554), (554, 555), (555, 557), (557, 558), (558, 559), (559, 560), (560, 561), (561, 562), (562, 563), (563, 564), (564, 565), (565, 566), (566, 567), (567, 568), (568, 569), (569, 570), (570, 571), (571, 572), (572, 573), (573, 574), (574, 575), (575, 576), (576, 577), (577, 578), (578, 579), (579, 580), (580, 581), (581, 582), (582, 583), (583, 584), (584, 585), (585, 586), (586, 587), (587, 588), (588, 589), (589, 590), (590, 591), (591, 592), (592, 593), (593, 594), (594, 595), (595, 596), (596, 597), (597, 598), (598, 599), (599, 600), (600, 601), (601, 603), (603, 604), (604, 606), (606, 607), (607, 608), (608, 609), (609, 610), (610, 611), (611, 612), (612, 613), (613, 614), (614, 615), (615, 616), (616, 617), (617, 618), (618, 619), (619, 620), (620, 621), (621, 622), (622, 623), (623, 624), (624, 625), (625, 626), (626, 627), (627, 631), (631, 632), (632, 633), (633, 637), (637, 638), (638, 639), (639, 640), (640, 641), (641, 642), (642, 643), (643, 644), (644, 645), (645, 646), (646, 647), (647, 648), (648, 649), (649, 650), (650, 651), (651, 652), (652, 653), (653, 657), (657, 658), (658, 659), (659, 660), (660, 662), (662, 663), (663, 664), (664, 665), (665, 666), (666, 667), (667, 668), (668, 669), (669, 670), (670, 671), (671, 672), (672, 673), (673, 674), (674, 675), (675, 676), (676, 677), (677, 678), (678, 679), (679, 680), (680, 681), (681, 682), (682, 683), (683, 684), (684, 685), (685, 686), (686, 687), (687, 688), (688, 689), (689, 690), (690, 691), (691, 692), (692, 693), (693, 694), (694, 695), (695, 696), (696, 697), (697, 698), (698, 699), (699, 700), (700, 701), (701, 702), (702, 703), (703, 704), (704, 705), (705, 706), (706, 707), (707, 708), (708, 709), (709, 710), (710, 714), (714, 715), (715, 716), (716, 717), (717, 719), (719, 720), (720, 721), (721, 722), (722, 723), (723, 724), (724, 725), (725, 726), (726, 727), (727, 728), (728, 729), (729, 730), (730, 731), (731, 732), (732, 733), (733, 734), (734, 735), (735, 736), (736, 737), (737, 738), (738, 739), (739, 740), (740, 741), (741, 742), (742, 743), (743, 744), (744, 745), (745, 746), (746, 747), (747, 748), (748, 749), (749, 750), (750, 751), (751, 752), (752, 753), (753, 754), (754, 755), (755, 756), (756, 757), (757, 758), (758, 759), (759, 760), (760, 761), (761, 762), (762, 763), (763, 767), (767, 768), (768, 769), (769, 770), (770, 772), (772, 773), (773, 774), (774, 775), (775, 776), (776, 777), (777, 778), (778, 779), (779, 780), (780, 781), (781, 782), (782, 783), (783, 785), (785, 786), (786, 787), (787, 788), (788, 789), (789, 790), (790, 791), (791, 792), (792, 793), (793, 795), (795, 796), (796, 797), (797, 798), (798, 799), (799, 800), (800, 801), (801, 802), (802, 803), (803, 804), (804, 805), (805, 806), (806, 807), (807, 808), (808, 809), (809, 810), (810, 811), (811, 812), (0, 0)], [(0, 0), (0, 1), (1, 2), (2, 3), (3, 4), (4, 5), (5, 6), (6, 7), (7, 8), (8, 9), (9, 10), (10, 11), (11, 12), (12, 13), (13, 14), (14, 15), (0, 0), (756, 757), (757, 758), (758, 759), (759, 760), (760, 761), (761, 762), (762, 763), (763, 767), (767, 768), (768, 769), (769, 770), (770, 772), (772, 773), (773, 774), (774, 775), (775, 776), (776, 777), (777, 778), (778, 779), (779, 780), (780, 781), (781, 782), (782, 783), (783, 785), (785, 786), (786, 787), (787, 788), (788, 789), (789, 790), (790, 791), (791, 792), (792, 793), (793, 795), (795, 796), (796, 797), (797, 798), (798, 799), (799, 800), (800, 801), (801, 802), (802, 803), (803, 804), (804, 805), (805, 806), (806, 807), (807, 808), (808, 809), (809, 810), (810, 811), (811, 812), (812, 813), (813, 814), (814, 815), (0, 0)]]\n"
     ]
    }
   ],
   "source": [
    "print(inputs['offset_mapping'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0, 0, 0, 0]\n"
     ]
    }
   ],
   "source": [
    "print(inputs['overflow_to_sample_mapping'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[None, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, None, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, None]\n",
      "300\n"
     ]
    }
   ],
   "source": [
    "print(inputs.sequence_ids())\n",
    "print(len(inputs.sequence_ids()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 [(0, 0), (0, 1), (1, 2), (2, 3), (3, 4), (4, 5), (5, 6), (6, 7), (7, 8), (8, 9), (9, 10), (10, 11), (11, 12), (12, 13), (13, 14), (14, 15), (0, 0), (0, 1), (1, 2), (2, 3), (3, 4), (4, 5), (5, 6), (6, 7), (7, 8), (8, 9), (9, 10), (10, 11), (11, 12), (12, 13), (13, 14), (14, 15), (15, 16), (16, 17), (17, 18), (18, 19), (19, 20), (20, 21), (21, 22), (22, 23), (23, 24), (24, 25), (25, 26), (26, 27), (27, 28), (28, 29), (29, 30), (30, 34), (34, 35), (35, 36), (36, 37), (37, 38), (38, 39), (39, 40), (40, 41), (41, 45), (45, 46), (46, 47), (47, 48), (48, 49), (49, 50), (50, 51), (51, 52), (52, 53), (53, 54), (54, 55), (55, 56), (56, 57), (57, 58), (58, 59), (59, 60), (60, 61), (61, 62), (62, 63), (63, 67), (67, 68), (68, 69), (69, 70), (70, 71), (71, 72), (72, 73), (73, 74), (74, 75), (75, 76), (76, 77), (77, 78), (78, 79), (79, 80), (80, 81), (81, 82), (82, 83), (83, 84), (84, 85), (85, 86), (86, 87), (87, 91), (91, 92), (92, 93), (93, 94), (94, 95), (95, 96), (96, 97), (97, 98), (98, 99), (99, 100), (100, 101), (101, 105), (105, 106), (106, 107), (107, 108), (108, 110), (110, 111), (111, 112), (112, 113), (113, 114), (114, 115), (115, 116), (116, 117), (117, 118), (118, 119), (119, 120), (120, 121), (121, 122), (122, 123), (123, 124), (124, 125), (125, 126), (126, 127), (127, 128), (128, 129), (129, 130), (130, 131), (131, 132), (132, 133), (133, 134), (134, 135), (135, 136), (136, 137), (137, 138), (138, 139), (139, 140), (140, 141), (141, 142), (142, 143), (143, 144), (144, 145), (145, 146), (146, 147), (147, 148), (148, 149), (149, 150), (150, 151), (151, 152), (152, 153), (153, 154), (154, 155), (155, 156), (156, 157), (157, 158), (158, 159), (159, 163), (163, 164), (164, 165), (165, 166), (166, 167), (167, 168), (168, 169), (169, 170), (170, 171), (171, 172), (172, 173), (173, 174), (174, 175), (175, 176), (176, 177), (177, 178), (178, 179), (179, 180), (180, 181), (181, 182), (182, 186), (186, 187), (187, 188), (188, 189), (189, 190), (190, 191), (191, 192), (192, 193), (193, 194), (194, 195), (195, 196), (196, 197), (197, 198), (198, 199), (199, 200), (200, 201), (201, 202), (202, 203), (203, 204), (204, 205), (205, 206), (206, 207), (207, 208), (208, 209), (209, 210), (210, 211), (211, 212), (212, 213), (213, 214), (214, 215), (215, 216), (216, 217), (217, 218), (218, 222), (222, 223), (223, 224), (224, 225), (225, 226), (226, 227), (227, 228), (228, 229), (229, 230), (230, 231), (231, 232), (232, 233), (233, 234), (234, 235), (235, 236), (236, 237), (237, 238), (238, 239), (239, 240), (240, 241), (241, 242), (242, 243), (243, 244), (244, 245), (245, 246), (246, 247), (247, 248), (248, 249), (249, 250), (250, 251), (251, 252), (252, 253), (253, 257), (257, 258), (258, 259), (259, 260), (260, 261), (261, 262), (262, 263), (263, 264), (264, 265), (265, 266), (266, 267), (267, 268), (268, 269), (269, 270), (270, 271), (271, 272), (272, 273), (273, 274), (274, 275), (275, 276), (276, 277), (277, 278), (278, 279), (279, 280), (280, 281), (281, 282), (282, 283), (283, 284), (284, 285), (285, 286), (286, 287), (287, 288), (288, 289), (289, 290), (290, 291), (291, 292), (292, 293), (293, 294), (294, 295), (295, 296), (296, 297), (297, 298), (298, 299), (299, 300), (300, 301), (301, 302), (302, 303), (303, 304), (304, 305), (305, 306), (306, 307), (307, 308), (308, 309), (309, 310), (0, 0)]\n",
      "0 {'text': ['1963年'], 'answer_start': [30]}\n",
      "0 [None, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, None, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, None]\n",
      "1 [(0, 0), (0, 1), (1, 2), (2, 3), (3, 4), (4, 5), (5, 6), (6, 7), (7, 8), (8, 9), (9, 10), (10, 11), (11, 12), (12, 13), (13, 14), (14, 15), (0, 0), (260, 261), (261, 262), (262, 263), (263, 264), (264, 265), (265, 266), (266, 267), (267, 268), (268, 269), (269, 270), (270, 271), (271, 272), (272, 273), (273, 274), (274, 275), (275, 276), (276, 277), (277, 278), (278, 279), (279, 280), (280, 281), (281, 282), (282, 283), (283, 284), (284, 285), (285, 286), (286, 287), (287, 288), (288, 289), (289, 290), (290, 291), (291, 292), (292, 293), (293, 294), (294, 295), (295, 296), (296, 297), (297, 298), (298, 299), (299, 300), (300, 301), (301, 302), (302, 303), (303, 304), (304, 305), (305, 306), (306, 307), (307, 308), (308, 309), (309, 310), (310, 311), (311, 312), (312, 313), (313, 314), (314, 315), (315, 316), (316, 317), (317, 318), (318, 319), (319, 320), (320, 321), (321, 325), (325, 326), (326, 327), (327, 328), (328, 329), (329, 330), (330, 331), (331, 332), (332, 333), (333, 334), (334, 335), (335, 336), (336, 337), (337, 338), (338, 339), (339, 340), (340, 341), (341, 342), (342, 343), (343, 344), (344, 345), (345, 346), (346, 347), (347, 348), (348, 349), (349, 350), (350, 351), (351, 352), (352, 353), (353, 354), (354, 355), (355, 356), (356, 360), (360, 361), (361, 362), (362, 363), (363, 364), (364, 365), (365, 366), (366, 367), (367, 368), (368, 369), (369, 370), (370, 371), (371, 372), (372, 373), (373, 374), (374, 375), (375, 376), (376, 377), (377, 378), (378, 379), (379, 380), (380, 381), (381, 382), (382, 383), (383, 384), (384, 385), (385, 386), (386, 387), (387, 388), (388, 390), (390, 391), (391, 392), (392, 393), (393, 394), (394, 395), (395, 396), (396, 397), (397, 398), (398, 399), (399, 400), (400, 401), (401, 402), (402, 403), (403, 404), (404, 405), (405, 406), (406, 407), (407, 408), (408, 409), (409, 410), (410, 411), (411, 412), (412, 413), (413, 414), (414, 415), (415, 416), (416, 417), (417, 418), (418, 419), (419, 420), (420, 421), (421, 422), (422, 424), (424, 425), (425, 426), (426, 427), (427, 428), (428, 429), (429, 430), (430, 431), (431, 432), (432, 433), (433, 434), (434, 435), (435, 436), (436, 437), (437, 438), (438, 439), (439, 440), (440, 441), (441, 442), (442, 443), (443, 444), (444, 445), (445, 446), (446, 447), (447, 448), (448, 449), (449, 450), (450, 451), (451, 452), (452, 453), (453, 454), (454, 455), (455, 456), (456, 457), (457, 458), (458, 459), (459, 460), (460, 461), (461, 462), (462, 463), (463, 464), (464, 465), (465, 466), (466, 467), (467, 468), (468, 469), (469, 470), (470, 471), (471, 472), (472, 473), (473, 474), (474, 475), (475, 476), (476, 477), (477, 478), (478, 479), (479, 480), (480, 481), (481, 482), (482, 483), (483, 484), (484, 485), (485, 486), (486, 487), (487, 488), (488, 489), (489, 490), (490, 491), (491, 492), (492, 493), (493, 494), (494, 495), (495, 499), (499, 500), (500, 501), (501, 502), (502, 503), (503, 504), (504, 505), (505, 506), (506, 507), (507, 508), (508, 509), (509, 510), (510, 511), (511, 512), (512, 513), (513, 514), (514, 516), (516, 517), (517, 518), (518, 519), (519, 520), (520, 521), (521, 522), (522, 523), (523, 524), (524, 525), (525, 526), (526, 527), (527, 528), (528, 529), (529, 530), (530, 531), (531, 532), (532, 533), (533, 534), (534, 535), (535, 536), (536, 537), (537, 538), (538, 539), (539, 540), (540, 541), (541, 542), (542, 543), (543, 544), (544, 545), (545, 546), (546, 547), (547, 548), (548, 552), (552, 553), (553, 554), (554, 555), (555, 557), (557, 558), (0, 0)]\n",
      "1 {'text': ['1963年'], 'answer_start': [30]}\n",
      "1 [None, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, None, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, None]\n",
      "2 [(0, 0), (0, 1), (1, 2), (2, 3), (3, 4), (4, 5), (5, 6), (6, 7), (7, 8), (8, 9), (9, 10), (10, 11), (11, 12), (12, 13), (13, 14), (14, 15), (0, 0), (503, 504), (504, 505), (505, 506), (506, 507), (507, 508), (508, 509), (509, 510), (510, 511), (511, 512), (512, 513), (513, 514), (514, 516), (516, 517), (517, 518), (518, 519), (519, 520), (520, 521), (521, 522), (522, 523), (523, 524), (524, 525), (525, 526), (526, 527), (527, 528), (528, 529), (529, 530), (530, 531), (531, 532), (532, 533), (533, 534), (534, 535), (535, 536), (536, 537), (537, 538), (538, 539), (539, 540), (540, 541), (541, 542), (542, 543), (543, 544), (544, 545), (545, 546), (546, 547), (547, 548), (548, 552), (552, 553), (553, 554), (554, 555), (555, 557), (557, 558), (558, 559), (559, 560), (560, 561), (561, 562), (562, 563), (563, 564), (564, 565), (565, 566), (566, 567), (567, 568), (568, 569), (569, 570), (570, 571), (571, 572), (572, 573), (573, 574), (574, 575), (575, 576), (576, 577), (577, 578), (578, 579), (579, 580), (580, 581), (581, 582), (582, 583), (583, 584), (584, 585), (585, 586), (586, 587), (587, 588), (588, 589), (589, 590), (590, 591), (591, 592), (592, 593), (593, 594), (594, 595), (595, 596), (596, 597), (597, 598), (598, 599), (599, 600), (600, 601), (601, 603), (603, 604), (604, 606), (606, 607), (607, 608), (608, 609), (609, 610), (610, 611), (611, 612), (612, 613), (613, 614), (614, 615), (615, 616), (616, 617), (617, 618), (618, 619), (619, 620), (620, 621), (621, 622), (622, 623), (623, 624), (624, 625), (625, 626), (626, 627), (627, 631), (631, 632), (632, 633), (633, 637), (637, 638), (638, 639), (639, 640), (640, 641), (641, 642), (642, 643), (643, 644), (644, 645), (645, 646), (646, 647), (647, 648), (648, 649), (649, 650), (650, 651), (651, 652), (652, 653), (653, 657), (657, 658), (658, 659), (659, 660), (660, 662), (662, 663), (663, 664), (664, 665), (665, 666), (666, 667), (667, 668), (668, 669), (669, 670), (670, 671), (671, 672), (672, 673), (673, 674), (674, 675), (675, 676), (676, 677), (677, 678), (678, 679), (679, 680), (680, 681), (681, 682), (682, 683), (683, 684), (684, 685), (685, 686), (686, 687), (687, 688), (688, 689), (689, 690), (690, 691), (691, 692), (692, 693), (693, 694), (694, 695), (695, 696), (696, 697), (697, 698), (698, 699), (699, 700), (700, 701), (701, 702), (702, 703), (703, 704), (704, 705), (705, 706), (706, 707), (707, 708), (708, 709), (709, 710), (710, 714), (714, 715), (715, 716), (716, 717), (717, 719), (719, 720), (720, 721), (721, 722), (722, 723), (723, 724), (724, 725), (725, 726), (726, 727), (727, 728), (728, 729), (729, 730), (730, 731), (731, 732), (732, 733), (733, 734), (734, 735), (735, 736), (736, 737), (737, 738), (738, 739), (739, 740), (740, 741), (741, 742), (742, 743), (743, 744), (744, 745), (745, 746), (746, 747), (747, 748), (748, 749), (749, 750), (750, 751), (751, 752), (752, 753), (753, 754), (754, 755), (755, 756), (756, 757), (757, 758), (758, 759), (759, 760), (760, 761), (761, 762), (762, 763), (763, 767), (767, 768), (768, 769), (769, 770), (770, 772), (772, 773), (773, 774), (774, 775), (775, 776), (776, 777), (777, 778), (778, 779), (779, 780), (780, 781), (781, 782), (782, 783), (783, 785), (785, 786), (786, 787), (787, 788), (788, 789), (789, 790), (790, 791), (791, 792), (792, 793), (793, 795), (795, 796), (796, 797), (797, 798), (798, 799), (799, 800), (800, 801), (801, 802), (802, 803), (803, 804), (804, 805), (805, 806), (806, 807), (807, 808), (808, 809), (809, 810), (810, 811), (811, 812), (0, 0)]\n",
      "2 {'text': ['1963年'], 'answer_start': [30]}\n",
      "2 [None, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, None, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, None]\n",
      "3 [(0, 0), (0, 1), (1, 2), (2, 3), (3, 4), (4, 5), (5, 6), (6, 7), (7, 8), (8, 9), (9, 10), (10, 11), (11, 12), (12, 13), (13, 14), (14, 15), (0, 0), (756, 757), (757, 758), (758, 759), (759, 760), (760, 761), (761, 762), (762, 763), (763, 767), (767, 768), (768, 769), (769, 770), (770, 772), (772, 773), (773, 774), (774, 775), (775, 776), (776, 777), (777, 778), (778, 779), (779, 780), (780, 781), (781, 782), (782, 783), (783, 785), (785, 786), (786, 787), (787, 788), (788, 789), (789, 790), (790, 791), (791, 792), (792, 793), (793, 795), (795, 796), (796, 797), (797, 798), (798, 799), (799, 800), (800, 801), (801, 802), (802, 803), (803, 804), (804, 805), (805, 806), (806, 807), (807, 808), (808, 809), (809, 810), (810, 811), (811, 812), (812, 813), (813, 814), (814, 815), (0, 0)]\n",
      "3 {'text': ['1963年'], 'answer_start': [30]}\n",
      "3 [None, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, None, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, None]\n",
      "[47, 0, 0, 0]\n",
      "[48, 0, 0, 0]\n"
     ]
    }
   ],
   "source": [
    "answers = [train_data[idx][\"answers\"] for idx in range(1)]\n",
    "start_positions = []\n",
    "end_positions = []\n",
    "\n",
    "for i, offset in enumerate(inputs[\"offset_mapping\"]):\n",
    "    print(i, offset)\n",
    "    sample_idx = inputs[\"overflow_to_sample_mapping\"][i]\n",
    "    answer = answers[sample_idx]\n",
    "    print(i, answer)\n",
    "    start_char = answer[\"answer_start\"][0]\n",
    "    end_char = answer[\"answer_start\"][0] + len(answer[\"text\"][0])\n",
    "    sequence_ids = inputs.sequence_ids(i)\n",
    "    print(i, sequence_ids)\n",
    "\n",
    "    # Find the start and end of the context\n",
    "    idx = 0\n",
    "    while sequence_ids[idx] != 1:\n",
    "        idx += 1\n",
    "    context_start = idx\n",
    "    while sequence_ids[idx] == 1:\n",
    "        idx += 1\n",
    "    context_end = idx - 1\n",
    "\n",
    "    # If the answer is not fully inside the context, label is (0, 0)\n",
    "    if offset[context_start][0] > start_char or offset[context_end][1] < end_char:\n",
    "        start_positions.append(0)\n",
    "        end_positions.append(0)\n",
    "    else:\n",
    "        # Otherwise it's the start and end token positions\n",
    "        idx = context_start\n",
    "        while idx <= context_end and offset[idx][0] <= start_char:\n",
    "            idx += 1\n",
    "        start_positions.append(idx - 1)\n",
    "\n",
    "        idx = context_end\n",
    "        while idx >= context_start and offset[idx][1] >= end_char:\n",
    "            idx -= 1\n",
    "        end_positions.append(idx + 1)\n",
    "\n",
    "print(start_positions)\n",
    "print(end_positions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_data_set = []\n",
    "data_file = file_path + 'cmrc2018_train.json'\n",
    "max_length=384\n",
    "real_max_lenght = 0\n",
    "index = 0\n",
    "\n",
    "def hand_new_data_set(data_item: map) -> map:\n",
    "    global real_max_lenght\n",
    "    global index\n",
    "    start_position=0\n",
    "    end_position=0\n",
    "    new_data_set_inner=[]\n",
    "\n",
    "    context = data_item[\"context\"]\n",
    "    question = data_item[\"question\"]\n",
    "    item_id = data_item[\"id\"]\n",
    "    answer = data_item[\"answers\"]\n",
    "\n",
    "    inputs = tokenizer(\n",
    "        question,\n",
    "        context,\n",
    "        max_length=max_length,\n",
    "        truncation=\"only_second\",\n",
    "        stride=50,\n",
    "        return_overflowing_tokens=True,\n",
    "        return_offsets_mapping=True\n",
    "    )\n",
    "\n",
    "    for i, offset in enumerate(inputs[\"offset_mapping\"]):\n",
    "        #print(i, offset)\n",
    "        #sample_idx = inputs[\"overflow_to_sample_mapping\"][i]\n",
    "        #answer = answers[sample_idx]\n",
    "        #print(i, answer)\n",
    "        start_char = answer[\"answer_start\"][0]\n",
    "        end_char = answer[\"answer_start\"][0] + len(answer[\"text\"][0])\n",
    "        sequence_ids = inputs.sequence_ids(i)\n",
    "        #print(i, sequence_ids)\n",
    "\n",
    "        # Find the start and end of the context\n",
    "        idx = 0\n",
    "        while sequence_ids[idx] != 1:\n",
    "            idx += 1\n",
    "        context_start = idx\n",
    "        while sequence_ids[idx] == 1:\n",
    "            idx += 1\n",
    "        context_end = idx - 1\n",
    "\n",
    "        # If the answer is not fully inside the context, label is (0, 0)\n",
    "        if offset[context_start][0] > start_char or offset[context_end][1] < end_char:\n",
    "            start_position=0\n",
    "            end_position=0\n",
    "        else:\n",
    "            # Otherwise it's the start and end token positions\n",
    "            idx = context_start\n",
    "            while idx <= context_end and offset[idx][0] <= start_char:\n",
    "                idx += 1\n",
    "            start_position=(idx - 1)\n",
    "\n",
    "            idx = context_end\n",
    "            while idx >= context_start and offset[idx][1] >= end_char:\n",
    "                idx -= 1\n",
    "            end_position=(idx + 1)\n",
    "        \n",
    "        ids=inputs[\"input_ids\"][i]\n",
    "        #print(tokenizer.decode(ids))\n",
    "        #print(offset)\n",
    "        if (context_end - context_start + 2 > max_length) :\n",
    "            print(context_end, \"-\", context_start, context_end - context_start)\n",
    "        \n",
    "        #print(offset[context_start], offset[context_end])\n",
    "        str_item_question = question\n",
    "        str_item_answer = context[int(offset[context_start][0]): int(offset[context_end][1])]\n",
    "        # print(str_item_question, str_item_answer)\n",
    "        # if len(str_item_answer) > max_length:\n",
    "        #     print(str_item_question, str_item_answer)\n",
    "\n",
    "        new_data_set_inner.append({\"id\":item_id, \"start_position\":start_position, \"end_position\": end_position, \"question\":str_item_question, \"answer\":str_item_answer})\n",
    "    return new_data_set_inner\n",
    "\n",
    "# sample = {\n",
    "#     \"id\": \"TRAIN_2323\",\n",
    "#     \"question\": \"\\\"十年最高\\\" 指什么？\", \n",
    "#     \"answers\":{'text': ['十年内各个月份最高流量值的平均数。'], 'answer_start': [883]}, \n",
    "#     \"context\": \"世界三大瀑布指巴西、阿根廷交界的伊瓜苏瀑布（世界上最宽）、东非赞比亚和津巴布韦交界的维多利亚瀑布以及美国加拿大交界的尼亚加拉瀑布。这三个瀑布无论从宽度、面积、流量都远超其他瀑布，并且都是跨国瀑布。世界排名第四的跨国瀑布是中国与越南交界的德天瀑布。1.^ a b c d e f Waterfalls Website 存取自二零零七年三月一日<br>2.^ a b c d e Commission on Dams website: \\\"Case Study — Kariba Dam-Zambezi River Basin\\\" Annex 13 & 14 Victoria Falls Mean Monthly Flows. 网站存取自二零零七年三月一日。此网站提供了每立方米/秒为单位的月平均流量(i.e., the total volume of water passing in each calendar month divided by the number of seconds in the month), 指水文学里冲量的标准，用作指示流量的季节变化。 A figure of around 9,000 m³/s (318,000 cu ft) is quoted by many websites for Victoria Falls but this is the mean maximum instantaneous rate, which is only achieved for a little amount of days per year. The figure of 536 million m³/minute (18.9 billion cu ft/min) on some websites (eg ZNTB) is an error for 536 million litres/minute (equivalent to 9100 m³/s or 142 million U.S. gallons/min). \\\"十年最高\\\" 是指十年内各个月份最高流量值的平均数。\", \n",
    "# }\n",
    "# hand_new_data_set(sample)\n",
    "\n",
    "with open(data_file, 'r', encoding='utf-8') as f:\n",
    "    json_data = json.load(f)\n",
    "    idx = 0\n",
    "    for article in json_data['data']:\n",
    "        title = article['title']\n",
    "        context = article['paragraphs'][0]['context']\n",
    "        for question in article['paragraphs'][0]['qas']:\n",
    "            q_id = question['id']\n",
    "            ques = question['question']\n",
    "            text = [ans['text'] for ans in question['answers']]\n",
    "            answer_start = [ans['answer_start'] for ans in question['answers']]\n",
    "            item_data = {\n",
    "                'id': q_id,\n",
    "                'title': title,\n",
    "                'context': context, \n",
    "                'question': ques,\n",
    "                'answers': {\n",
    "                    'text': text,\n",
    "                    'answer_start': answer_start\n",
    "                }\n",
    "            }\n",
    "            #hand_new_data_set(item_data)\n",
    "            new_data_set.extend(hand_new_data_set(item_data))\n",
    "\n",
    "# print(real_max_lenght)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "17846 {'id': 'TRAIN_186_QUERY_0', 'start_position': 47, 'end_position': 48, 'question': '范廷颂是什么时候被任为主教的？', 'answer': '范廷颂枢机（，），圣名保禄·若瑟（），是越南罗马天主教枢机。1963年被任为主教；1990年被擢升为天主教河内总教区宗座署理；1994年被擢升为总主教，同年年底被擢升为枢机；2009年2月离世。范廷颂于1919年6月15日在越南宁平省天主教发艳教区出生；童年时接受良好教育后，被一位越南神父带到河内继续其学业。范廷颂于1940年在河内大修道院完成神学学业。范廷颂于1949年6月6日在河内的主教座堂晋铎；及后被派到圣女小德兰孤儿院服务。1950年代，范廷颂在河内堂区创建移民接待中心以收容到河内避战的难民。1954年，法越战争结束，越南民主共和国建都河内，当时很多天主教神职人员逃至越南的南方，但范廷颂仍然留在河内。翌年管理圣若望小修院；惟在1960年因捍卫修院的自由、自治及拒绝政府在修院设政治课的要求而被捕。1963年4月5日，教宗任命范廷颂为天主教北宁教区主教，同年8月15日就任；其牧铭为「我信'}\n"
     ]
    }
   ],
   "source": [
    "\n",
    "print(len(new_data_set), new_data_set[0])\n",
    "filename=file_path +\"chunck_train.json\"\n",
    "with open(filename, 'w') as j: \n",
    "    j.write(json.dumps(new_data_set))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename=file_path +\"chunck_train1.json\"\n",
    "data1 = [{\"asion\":\"0000001\",\"name\":\"wuqq\"},{\"asion\":\"0000002\",\"name\":\"zxw\"}]\n",
    "with open(filename, 'w') as j: \n",
    "    j.write(json.dumps(data1))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
